/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package uit.ieproject.crawler;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import java.awt.Color;
import java.awt.image.BufferedImage;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.imageio.ImageIO;
import jvntokenizer.PennTokenizer;
import uit.ieproject.presentation.MainFrame;
import uit.ieproject.presentation.ProgressDownloadFrame;
import uit.ieproject.utilites.FileUtil;
import uit.ieproject.utilites.ImageUtil;
import uit.ieproject.utilites.TextpaneUtil;

/**
 *
 * @author RedDevils
 */
public class MyCrawler extends WebCrawler {

    private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g" + "|png|tiff?|mid|mp2|mp3|mp4"
            + "|wav|avi|mov|mpeg|ram|m4v|pdf" + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");
    private float floatValuePBar = 0;
    private boolean isExistImage = false;
    private int numFiles;

    public MyCrawler() {
        numFiles = MainFrame.downloadListModel.getSize();
    }

    /**
     * You should implement this function to specify whether the given url
     * should be crawled or not (based on your crawling logic).
     */
    @Override
    public boolean shouldVisit(WebURL url) {
        String href = url.getURL().toLowerCase();
        ArrayList<String> listUrl = new ArrayList<>();
        listUrl = MainFrame.arrlistLink;
        for (int i = 0; i < listUrl.size(); i++) {
            return !FILTERS.matcher(href).matches() && href.contains(listUrl.get(i));
        }
        return false;
    }

    /**
     * This function is called when a page is fetched and ready to be processed
     * by your program.
     */
    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();
        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            String html = htmlParseData.getHtml();

            try {
                List<WebURL> links = htmlParseData.getOutgoingUrls();

                System.out.println("Text length: " + text.length());
                System.out.println("Html length: " + html.length());
                System.out.println("Number of outgoing links: " + links.size());

                ArrayList<String> document;
                String str = PennTokenizer.parseHTML(html);
                //get title
                String title = getStrTitleWithPattern(MainFrame.title, str);
                if (title.length() > 0) {

                    String tit = PennTokenizer.tokenize(title);
                    String filename = PennTokenizer.tokenizeFileName(tit.trim());
                    //get document
                    document = getArrStrWithPattern(MainFrame.paragraph, str);
                    if (MainFrame.downloadListModel.getSize() != 0) {
                        float increase = (float) 100 / numFiles;
                        floatValuePBar += increase;
                        ProgressDownloadFrame.pbStatus.setValue(Math.round(floatValuePBar));
                        ProgressDownloadFrame.pbStatus.setStringPainted(true);
                    }
                    if (document.size() > 0) {
                        TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "-------------------------------------------------\n", Color.black);
                        TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "Đang lưu file...." + filename + "\n", Color.blue);
                        FileUtil.saveTextToFile(MainFrame.directory, filename, "txt", document, false);
                        TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "Lưu file...." + filename + "\n", Color.BLACK);
                        //MyControllerThread.model.removeElement(filename);
                        MainFrame.downloadListModel.removeElement(filename);
                    } else {
                        TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "-------------------------------------------------\n", Color.black);
                        TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "Không tải được nhân vật " + filename + "\n", Color.red);
                    }

                    //get image
                    String img = getStrWithPattern(MainFrame.image, str);
                    if (img.length() > 0) {
                        ArrayList<String> listURLImage = new ArrayList<>();
                        String urlImage = getStrWithPatternGroup1(Patterns.STR_URL, img);
                        listURLImage.add(urlImage);
                        FileUtil.saveTextToFile(MainFrame.directory, "infoImage", "txt", listURLImage, true);
                        isExistImage = testExistImage("infoImage.txt", urlImage);
                        if (urlImage.length() > 0 && !isExistImage) {
                            TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "Đang lưu hình ảnh..." + urlImage + "\n", Color.blue);
                            ImageUtil.saveImageFromWeb(urlImage, MainFrame.directory, filename, "jpg");
                            TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "Lưu hình ảnh thành công" + "\n", Color.black);
                        } else {
                            TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "Hình ảnh nhân vật " + filename + " không được lưu\n", Color.red);
                        }
                    }
                }

            } catch (Exception ex) {
                Logger.getLogger(MyCrawler.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        TextpaneUtil.setTextPane(ProgressDownloadFrame.tpStatus, "==========================================\n", Color.black);
    }

    public ArrayList<String> getArrStrWithPattern(String regex, String inputStr) {
        ArrayList<String> list = new ArrayList<>();
        String returnStr = "";
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(inputStr);
        //get only first occurrence
        while (matcher.find()) {
            returnStr = matcher.group();
            //remove html tags
            list.add(PennTokenizer.tokenize(returnStr));
        }
        return list;
    }

    public String getStrWithPattern(String regex, String inputStr) {
        String returnStr = "";
        Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE);
        Matcher matcher = pattern.matcher(inputStr);
        //get only first occurrence
        while (matcher.find()) {
            returnStr = matcher.group();
        }
        return returnStr;
    }

    public String getStrTitleWithPattern(String regex, String inputStr) {
        String returnStr = "";
        Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE);
        Matcher matcher = pattern.matcher(inputStr);
        //get only first occurrence
        while (matcher.find()) {
            returnStr = matcher.group();
            break;
        }
        return returnStr;
    }

    public String getStrWithPatternGroup1(String regex, String inputStr) {
        String returnStr = "";
        Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE);
        Matcher matcher = pattern.matcher(inputStr);
        //get only first occurrence
        while (matcher.find()) {
            returnStr = matcher.group(1);
        }
        return returnStr;
    }

    public boolean testExistImage(String fileOpen, String addNewURL) throws IOException {
        boolean isExist = false;
        try {
            ArrayList<String> readListURL = new ArrayList<>();
            InputStream fis;
            BufferedReader br;
            String line;

            fis = new FileInputStream(MainFrame.directory + fileOpen);
            br = new BufferedReader(new InputStreamReader(fis, Charset.forName("UTF-8")));
            while ((line = br.readLine()) != null) {
                // Deal with the line
                readListURL.add(line);
            }
            int lengthURL = readListURL.size() - 1;
            for (int i = 0; i < lengthURL; i++) {
                if (readListURL.get(i).equals(addNewURL)) {
                    isExist = true;
                    break;
                }
            }
        } catch (FileNotFoundException ex) {
            Logger.getLogger(MyCrawler.class.getName()).log(Level.SEVERE, null, ex);
        }
        return isExist;
    }
}
